# -*- coding: utf-8 -*-
import scrapy
import json
from scrapy.http.response.html import HtmlResponse
from ..items import ItemType,JiayuanUserInfo

class JiayuanUserListSpider(scrapy.Spider):
    name = 'jiayuan_user_list'
    allowed_domains = ['jiayuan.com']
    #start_urls = ['https://search.jiayuan.com/v2/search_v2.php?sex=f&key=&sn=default&sv=1&p=1&f=&listStyle=bigPhoto&pri_uid=0&jsversion=v5']
    pageIndex = 1
    formdata = {
        'kw': 'wolf',
        'sex': 'f',
        'sn': 'default',
        'sv': '1',
        'p': '1',
        'listStyle': 'bigPhoto',
        'pri_uid ': '0',
        'jsversion': 'v5'
    }
    post_url = 'https://search.jiayuan.com/v2/search_v2.php'
    headers = {
        'Content-Type': 'application/json;charset=utf-8'
    }
    def start_requests(self):
        # 请求的url

        # post请求参数




        # 发送post请求
        yield scrapy.FormRequest(url=self.post_url ,method = 'POST',formdata=self.formdata, headers=self.headers,callback=self.parse, dont_filter = True)

    def parse(self, response:HtmlResponse):
        str_body = response.body
        if isinstance(response.body, bytes):
            str_body = bytes.decode(response.body).replace('##jiayser##', '', 3).replace('//', '', 3)
        resjson = json.loads(str_body)

        info = JiayuanUserInfo()
        for item in resjson['userInfo']:
            info['itemName'] = ItemType.JIAY_USER_LIST_ITEM.value
            info['uid'] = item['uid']
            info['realUid'] = item['realUid']
            info['nickName'] = item['nickname']
            info['sex'] = item['sexValue']
            info['age'] = item['age']
            info['marriage'] = item['marriage']
            info['height'] = item['height']
            info['education'] = item['education']
            info['image'] = item['image']
            info['workLocation'] = item['work_location']

            if 'xjhykj_f.jpg' not in info['image']:
                yield info

        pageTotal =  resjson['pageTotal']
        if (self.pageIndex < pageTotal):
            self.pageIndex = self.pageIndex + 1
            self.formdata['p'] = str(self.pageIndex)
            yield scrapy.FormRequest(
                url=self.post_url,
                method='POST',
                formdata=self.formdata,
                headers=self.headers,
                callback=self.parse, dont_filter=True)

        else:
            print('分页爬虫完成')
        pass
